{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# 3.使用 lightGBM 预测音乐推荐结果     模型训练"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [],
   "source": [
    "# 首先 import 必要的模块\n",
    "import pandas as pd \n",
    "import numpy as np\n",
    "import pickle as pk\n",
    "import seaborn as sns\n",
    "import matplotlib.pyplot as plt\n",
    "import math\n",
    "import scipy.io as sio\n",
    "import scipy.sparse as ss\n",
    "from sklearn.preprocessing import LabelEncoder\n",
    "from sklearn.model_selection import GridSearchCV\n",
    "%matplotlib inline"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "import lightgbm as lgbm\n",
    "from lightgbm.sklearn import LGBMClassifier\n",
    "from lightgbm.sklearn import LGBMRegressor"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "data_path = '../Data/'  # 文件路径\n",
    "model_path = '../model/' # 模型路径"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "# 载入标签数据\n",
    "with open(model_path + 'target_list.pkl','rb') as fr:\n",
    "    train_Y = pk.load(fr)\n",
    "fr.close()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "# with open(model_path + 'data_all_train_lgbm_v1.pkl','wb') as fw:\n",
    "#     pk.dump(train_X,fw)\n",
    "# fw.close()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "# 载入经过处理的训练数据\n",
    "with open(model_path + 'data_all_train_lgbm_v1.pkl','rb') as fr:\n",
    "    train_X = pk.load(fr)\n",
    "fr.close()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## LightGBM超参数调优"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "LightGBM的主要的超参包括：\n",
    "1. 树的数目n_estimators 和 学习率 learning_rate\n",
    "2. 树的最大深度max_depth 和 树的最大叶子节点数目num_leaves（注意：XGBoost只有max_depth，LightGBM采用叶子优先的方式生成树，num_leaves很重要，设置成比 2^max_depth 小）\n",
    "3. 叶子结点的最小样本数:min_data_in_leaf(min_data, min_child_samples)\n",
    "4. 每棵树的列采样比例：feature_fraction/colsample_bytree\n",
    "5. 每棵树的行采样比例：bagging_fraction （需同时设置bagging_freq=1）/subsample\n",
    "6. 正则化参数lambda_l1(reg_alpha), lambda_l2(reg_lambda)\n",
    "7. 两个非模型复杂度参数，但会影响模型速度和精度。可根据特征取值范围和样本数目修改这两个参数\n",
    "1）特征的最大bin数目max_bin：默认255；\n",
    "2）用来建立直方图的样本数目subsample_for_bin：默认200000。\n",
    "\n",
    "对n_estimators，用LightGBM内嵌的cv函数调优，因为同XGBoost一样，LightGBM学习的过程内嵌了cv，速度极快。\n",
    "其他参数用GridSearchCV"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    "MAX_ROUNDS = 10000"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### 相同的交叉验证分组"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [],
   "source": [
    "# prepare cross validation\n",
    "from sklearn.model_selection import StratifiedKFold\n",
    "\n",
    "kfold = StratifiedKFold(n_splits=3, shuffle=True, random_state=3)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [],
   "source": [
    "# best_n_estimators = len(cv_result['auc-mean'])\n",
    "# print(\"best estimator's num: %d\" % n_estimators)\n",
    "best_n_estimators = 9431"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### 降低学习率，使用 2_1得到的最佳参数 再次计算最佳 n_estimators"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "CPU times: user 0 ns, sys: 0 ns, total: 0 ns\n",
      "Wall time: 5.25 µs\n"
     ]
    }
   ],
   "source": [
    "%%time\n",
    "#直接调用 lightgbm 内嵌的交叉验证(cv)，可对连续的 n_estimators 参数进行快速交叉验证\n",
    "#而GridSearchCV只能对有限个参数进行交叉验证，且速度相对较慢\n",
    "def get_n_estimators(params , train_X , train_Y , early_stopping_rounds=10):\n",
    "    lgbm_params = params.copy()\n",
    "     \n",
    "    lgbmtrain = lgbm.Dataset(train_X , train_Y)\n",
    "     \n",
    "    #num_boost_round为弱分类器数目，下面的代码参数里因为已经设置了early_stopping_rounds\n",
    "    #即性能未提升的次数超过过早停止设置的数值，则停止训练\n",
    "    cv_result = lgbm.cv(lgbm_params , lgbmtrain , num_boost_round=MAX_ROUNDS , nfold=3,  metrics='auc' , early_stopping_rounds=early_stopping_rounds,seed=3 )\n",
    "     \n",
    "    return cv_result"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "CPU times: user 1d 1h 27min, sys: 33min 14s, total: 1d 2h 15s\n",
      "Wall time: 4h 26min 24s\n"
     ]
    }
   ],
   "source": [
    "%%time\n",
    "params = {'boosting_type': 'gbdt',\n",
    "          'objective': 'regression',\n",
    "          'n_jobs': 6,\n",
    "          'learning_rate': 0.01,\n",
    "          'min_child_samples':70, # 每个叶子节点的最小样本数目\n",
    "          # 'n_estimators':best_n_estimators,\n",
    "          'max_depth': 7,\n",
    "          'max_bin': 64, # 稀疏离散特征一共有167维，先不设置很大，提高训练速度\n",
    "          'subsample': 0.7,\n",
    "          'bagging_freq': 1,\n",
    "          'colsample_bytree': 0.7,\n",
    "          'num_leaves': 30,\n",
    "         }\n",
    "\n",
    "cv_result = get_n_estimators(params , train_X , train_Y)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.7026156629288195"
      ]
     },
     "execution_count": 22,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "cv_result['auc-mean'][9999]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.6936913097430314"
      ]
     },
     "execution_count": 15,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "np.mean(cv_result['auc-mean'])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "10000"
      ]
     },
     "execution_count": 16,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "len(cv_result['auc-mean'])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# 用所有的训练数据，使用最新的训练参数训练模型"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "best_n_estimators = 3000"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "params = {'boosting_type': 'gbdt',\n",
    "          'objective': 'regression',\n",
    "          'n_jobs': 6,\n",
    "          'learning_rate': 0.01,\n",
    "          'min_child_samples':70, # 每个叶子节点的最小样本数目\n",
    "          'n_estimators':best_n_estimators,\n",
    "          'max_depth': 7,\n",
    "          'max_bin': 64, # 稀疏离散特征一共有167维，先不设置很大，提高训练速度\n",
    "          'subsample': 0.7,\n",
    "          'bagging_freq': 1,\n",
    "          'colsample_bytree': 0.7,\n",
    "          'num_leaves': 30,\n",
    "         }\n",
    "lg = LGBMRegressor(silent=False,  **params)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "\n",
    "\n",
    "\n",
    "# num_leaves_s = range(10,50,20) #50,60,70,80\n",
    "# tuned_parameters = dict( num_leaves = num_leaves_s)\n",
    "lg.fit(train_X,train_Y)\n",
    "# print('tuned_parameters',tuned_parameters)\n",
    "# grid_search_num_leaves = GridSearchCV(lg, n_jobs=6, param_grid=tuned_parameters, cv = kfold, scoring=\"neg_log_loss\", verbose=10, refit = False)\n",
    "# grid_search_num_leaves.fit(train_X , train_Y)\n",
    "#grid_search.best_estimator_"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [],
   "source": [
    "try:\n",
    "    with open(model_path + 'lg_v2_2_4.pkl','wb') as fw:\n",
    "        pk.dump(lg,fw)\n",
    "    fw.close()\n",
    "except Exception as e:\n",
    "    print('dump lg_v2_2_4.pkl error,',e)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "# # examine the best model\n",
    "# print(-grid_search.best_score_)\n",
    "# print(grid_search.best_params_)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "# # plot CV误差曲线\n",
    "# test_means = grid_search.cv_results_[ 'mean_test_score' ]\n",
    "# test_stds = grid_search.cv_results_[ 'std_test_score' ]\n",
    "# train_means = grid_search.cv_results_[ 'mean_train_score' ]\n",
    "# train_stds = grid_search.cv_results_[ 'std_train_score' ]\n",
    "\n",
    "# n_leafs = len(num_leaves_s)\n",
    "\n",
    "# x_axis = num_leaves_s\n",
    "# plt.plot(x_axis, -test_means)\n",
    "# #plt.errorbar(x_axis, -test_means, yerr=test_stds,label = ' Test')\n",
    "# #plt.errorbar(x_axis, -train_means, yerr=train_stds,label = ' Train')\n",
    "# plt.xlabel( 'num_leaves' )\n",
    "# plt.ylabel( 'Log Loss' )\n",
    "# plt.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# test_means"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.3"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}
